import com.alibaba.fastjson.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by msi on 2018/6/6 0006.
 * 针对单个城市页面爬取
 */
public class FangTianXiaPageProcess implements PageProcessor {
    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    private static int count = 0;

    private static String currentId = null;
    private static String currentName = null;
    private static String city = null;

    public void process(Page page) {
        String pageUrl = page.getUrl().toString();
        String newCode = page.getHtml().regex("newcode=([0-9]+)").get();

        String mainBuilding = page.getHtml().regex("(mainBuilding)").get();
        //由于在iframe中 抓取时候 不会负责解析所以需要再加一层 【二手房 暂定】
        if (newCode != null) {
            String title = page.getHtml().getDocument().title();
            String dituUrl = "http://ditu.fang.com/?c=channel&a=xiaoquNew&newcode=" + newCode + "&city=" + city;
            page.addTargetRequest(dituUrl);
            System.out.println(currentId + "|" + currentName + "添加【地图】链接信息|【" + title + "】|" + dituUrl);
        }

        //抓取地图信息
        if (mainBuilding != null) {
            count++;
            String data = page.getHtml().regex("\\{.+?\\}").toString();
            System.out.println(currentId + "|" + currentName + "小区【详细】信息|" + dataProcess(data));
        }

        if (pageUrl.indexOf("newhouse") < 0 && pageUrl.indexOf("esf") < 0 && mainBuilding == null && newCode == null) {//如果是刚进首页 抓取链接进入链接池
            String tiltle = page.getHtml().getDocument().title();
            if (tiltle.contains(currentName)) {//必须要一致 不然海外 澳大利亚没有 和北京不一致循环bug
                String newhouseurl = page.getHtml().xpath("//*[@id=\"dsy_H01_03\"]/div[1]/a").links().get();
                String esfurl = page.getHtml().xpath("//*[@id=\"dsy_H01_04\"]/div[1]/a").links().get();
                page.addTargetRequest(newhouseurl);
            }
        } else if (pageUrl.indexOf("newhouse") > -1 && pageUrl.indexOf("esf") < 0 && mainBuilding == null && newCode == null) {//如果是新房
            getFangList(page);
        } else if (pageUrl.indexOf("newhouse") < 0 && pageUrl.indexOf("esf") > -1 && mainBuilding == null && newCode == null) { //如果是二手房

        }

    }

    public String dataProcess(String data) {
        JSONObject jsonData = JSONObject.parseObject(data);

        //小区代码
        String newCode = jsonData.containsKey("newCode") ? jsonData.get("newCode").toString() : "";

        //小区信息主页
        String houseurl = jsonData.containsKey("houseurl") ? jsonData.get("houseurl").toString() : "";

        //小区名称
        String title = jsonData.containsKey("title") ? jsonData.get("title").toString() : "";

        //地址
        String address = jsonData.containsKey("address") ? jsonData.get("address").toString() : "";

        //联系方式
        String tel = jsonData.containsKey("tel") ? jsonData.get("tel").toString() : "";

        //单价
        String price_num = jsonData.containsKey("price_num") ? jsonData.get("price_num").toString() : "";

        //单价单元
        String price_unit = jsonData.containsKey("price_unit") ? jsonData.get("price_unit").toString() : "";


        //单价类型
        String price_type = jsonData.containsKey("price_type") ? jsonData.get("price_type").toString() : "";

        //单价日期
        String price_date = jsonData.containsKey("price_date") ? jsonData.get("price_date").toString() : "";


        //baidu_coord_x 与mapx是一样的 还是转一下坐标系
        String mapx = jsonData.containsKey("mapx") ? jsonData.get("mapx").toString() : "";
        String mapy = jsonData.containsKey("mapy") ? jsonData.get("mapy").toString() : "";

        double x = mapx != "" ? Double.valueOf(mapx).doubleValue() : 0.0;
        double y = mapy != "" ? Double.valueOf(mapy).doubleValue() : 0.0;

        //绿化率
        String virescencerate = jsonData.containsKey("virescencerate") ? jsonData.get("virescencerate").toString() : "";

        //容积率
        String dimension = jsonData.containsKey("dimension") ? jsonData.get("dimension").toString() : "";

        //停车位
        String parkdesc = jsonData.containsKey("parkdesc") ? jsonData.get("parkdesc").toString() : "";

        //物业费
        String propertyfee = jsonData.containsKey("propertyfee") ? jsonData.get("propertyfee").toString() : "";

        //物业费单位
        String propfeetype = jsonData.containsKey("propfeetype") ? jsonData.get("propfeetype").toString() : "";

        //户数
        String parttotaldoor = jsonData.containsKey("parttotaldoor") ? jsonData.get("parttotaldoor").toString() : "";

        //完成日期
        String finishdate = jsonData.containsKey("finishdate") ? jsonData.get("finishdate").toString() : "";

        //建筑面积
        String purposeArea = jsonData.containsKey("purposeArea") ? jsonData.get("purposeArea").toString() : "";

        //最小户型
        String minarea = jsonData.containsKey("minarea") ? jsonData.get("minarea").toString() : "";

        //最大户型
        String maxarea = jsonData.containsKey("maxarea") ? jsonData.get("maxarea").toString() : "";

        //楼栋id总计 数据有问题
        String dongid = jsonData.containsKey("dongid") ? jsonData.get("dongid").toString() : "";
        dongid = "" + dongid.split(",").length;
        String dataResult = currentId + "|" + currentName + "|" + title + newCode + "|" + houseurl + "|" + "|" + address + "|" + tel + "|" + price_num + "|" + price_unit + "|" + price_type +
                "|" + price_date + "|" + mapx + "|" + mapy + "|" + x + "|" + y + "|" + virescencerate + "|" + dimension + "|" + parkdesc + "|" + propertyfee +
                "|" + propfeetype + "|" + parttotaldoor + "|" + finishdate + "|" + purposeArea + "|" + minarea + "|" + maxarea + "|" + dongid;
        try {
            writeTxt("E:\\fangs.txt", dataResult);
        } catch (Exception e) {
            System.out.print("【error】写入文件有问题请查找原因");
            e.printStackTrace();
        }
        return dataResult;
    }

    public Site getSite() {
        return site;
    }


    public void writeTxt(String fileName, String content) {
        BufferedWriter out = null;
        try {
            out = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(fileName, true)));
            out.write(content + "\r\n");
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                out.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }


    public void getFangList(Page page) {

        if (page.getHtml().regex("(下一页)").get() != null) {//添加下一页的链接到链接池
            List<String> nextPageUrls = page.getHtml().xpath("//a[@class='next']/@href").all();
            if (nextPageUrls != null && nextPageUrls.size() > 0) {
                page.addTargetRequest(nextPageUrls.get(nextPageUrls.size() - 1));
                System.out.println(currentId + "|" + currentName + "添加【下一页】链接信息|" + nextPageUrls.get(nextPageUrls.size() - 1));
            }
        } else if (page.getHtml().regex("(尾页)").get() != null) {//添加尾页的链接到链接池 虽然好像没用到
            String url = page.getUrl().toString();
            if (url.indexOf("b9") > -1) {
                String nextPageCount = url.substring(url.indexOf("b9") + 2, url.length() - 1);
                url = url.replace("b9" + nextPageCount, "b9" + (Integer.valueOf(nextPageCount) + 1));
                page.addTargetRequest(url);
                System.out.println(currentId + "|" + currentName + "添加【尾页】链接信息|" + url);

            }
        }

        List<String> items = page.getHtml().xpath("//div[@class='nhouse_list']").links().regex("(http://\\w+\\.fang\\.com/*)").all();
        if (items != null && items.size() > 0) {
            //去重 正常每页是20个 将小区首页链接添加链接池中
            List<String> itemsWithoutDup = new ArrayList<String>(new HashSet<String>(items));
            page.addTargetRequests(itemsWithoutDup);
            System.out.println(currentId + "|" + currentName + "添加【小区列表】链接信息|【" + itemsWithoutDup.size() + "个】");
        }

    }

    public static void main(String[] args) {
        long startTime, endTime;
        System.out.println("开始爬取...");
        startTime = System.currentTimeMillis();
        StringBuilder result = new StringBuilder();
//        String[] haiwai = {"海外", "美国", "日本", "澳大利亚", "英国",
//                "马来西亚", "泰国", "加拿大", "塞浦路斯", "新加坡", "希腊", "迪拜", "新西兰", "澳门", "香港", "台湾"};
//        List<String> haiwais = Arrays.asList(haiwai);
        try {
            BufferedReader br = new BufferedReader(new FileReader(new File(ClassLoader.getSystemResource("cities.txt").getPath())));//构造一个BufferedReader类来读取文件
            String s = null;
            Pattern patternUrl = Pattern.compile("//\\w*");
            while ((s = br.readLine()) != null) {//使用readLine方法，一次读一行
                count = 0;
                String[] value = s.split("\\|");
                String id = value[0];
                String name = value[1];
                String url = value[2];
                currentId = id;
                currentName = name;
//                if (haiwais.contains(currentName)) {
//                    continue;
//                }
                Matcher matcherUrl = patternUrl.matcher(url);
                if (matcherUrl.find())
                    city = matcherUrl.group().substring(2, matcherUrl.group().length());
                System.out.println(id + "|" + name + "爬取中...");
                Spider.create(new FangTianXiaPageProcess()).addUrl(url).thread(100).run();
                endTime = System.currentTimeMillis();
                System.out.println(id + "|" + name + "爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了" + count + "条记录");
            }
            br.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
